
import sys, os, re
import urllib
import StringIO
from lxml import etree
from dateutil import parser as dateparser

from baseHandler import *

parser = etree.HTMLParser()
baseuri = "http://webarchive.nationalarchives.gov.uk/*/"


class UkHandler(BaseProxyHandler):


    def fetch_changes(self, req, requri, dt=None):

        iauri = baseuri + requri

        try:
            fh = urllib.urlopen(iauri)
        except:
            return self.error("Couldn't retrieve data from %s" % iauri, req, status=404)
        data = fh.read()
        fh.close()
        try:
            dom = etree.parse(StringIO.StringIO(data), parser)
        except:
            return self.error("Couldn't parse data from %s" % iauri, req, status=500)

        alist = dom.xpath('//div[@id="tblcontainer"]//a')
        if not alist:
            return self.error("URL (%s) doesn't exist in IA" % iauri, req, status=404)            

        changes = []

        for a in alist:
            if a.attrib.has_key('name'):
                continue
            try:
                dtobj = dateparser.parse(a.text + " 00:00:00 GMT")
            except:
                continue
            loc = a.attrib['href']
            if loc[0] == '/':
                loc = "http://webarchive.nationalarchives.gov.uk" + loc
            if not loc.startswith('http://webarchive.nationalarchives.gov.uk/'):
                continue

            info = {'last' : dtobj, 'obs': 1}
            changes.append((dtobj, loc, info))
        return changes
                

def handler(req):
    hdlr = UkHandler('uk')
    return basehandler(req, hdlr)
